{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "import requests\n",
    "import re\n",
    "from lxml import etree\n",
    "import tianya_csv\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import random\n",
    "from time import sleep\n",
    "import time\n",
    "from faker import Factory\n",
    "from multiprocessing.dummy import Pool as TreadPool\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import unquote\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 获取html文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getHTMLResponse(url):\n",
    "    fc = Factory.create()\n",
    "    headers = {\n",
    "        'Connection': 'keep-alive',\n",
    "        'Cache-Control': 'max-age=0',\n",
    "        'Upgrade-Insecure-Requests': '1',\n",
    "        'User-Agent': fc.user_agent(),\n",
    "        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',\n",
    "        'Sec-Fetch-Site': 'same-origin',\n",
    "        'Sec-Fetch-Mode': 'navigate',\n",
    "        'Sec-Fetch-User': '?1',\n",
    "        'Sec-Fetch-Dest': 'document',\n",
    "        'Referer': 'https://search.cn-ki.net/search?keyword=%E7%88%AC%E8%99%AB&db=CFLS&p=13',\n",
    "        'Accept-Language': 'zh-CN,zh;q=0.9',\n",
    "        'cookie': '_T_WM=7cd2fe00a46ec406f450f4ad4764df2b; SUB=_2A25yHhL1DeRhGeNL6FUZ8inIyD-IHXVR4L69rDV6PUNbktANLWXBkW1NSRtw-UgxtiIVZEtYWqy18AK3UycSktDK; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WW9ciTfJHi3v88u56ZUFhyf5JpX5KzhUgL.Fo-fe0MReoMXe0e2dJLoIpzLxKqL1h2LB.2LxKqLBK2L1K2t; SUHB=0sqBYxW86J6g_p; SSOLoginState=1595564709; ALF=1598156709',\n",
    "    }\n",
    "    try:\n",
    "        r = requests.get(url,headers=headers,timeout=15)\n",
    "        r.raise_for_status()\n",
    "        r.encoding = r.apparent_encoding\n",
    "        # 无框架延迟处理\n",
    "#         random_time = random.randint(3,5)\n",
    "#         sleep(random_time)\n",
    "        return r\n",
    "    except:\n",
    "        print(\"获取html失败\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 获取天涯搜索页面列表信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData(content,csv_file):\n",
    "    try:\n",
    "        tree_node = etree.HTML(content)\n",
    "        doc_ul_lis = tree_node.xpath('//div[@class=\"searchListOne\"]/ul/li')\n",
    "        for li in doc_ul_lis:\n",
    "            # 标题\n",
    "            title = li.xpath('./div/h3/a')\n",
    "            if title == []:\n",
    "                break\n",
    "            else:\n",
    "                title = title[0].xpath('string(.)').strip()\n",
    "                print(title)\n",
    "            # 板块\n",
    "            plate = li.xpath('./p/a[1]/text()')[-1]\n",
    "#             print(plate)\n",
    "            # 作者\n",
    "            author = li.xpath('./p/a[2]/text()')[-1]\n",
    "#             print(author)\n",
    "            # 时间\n",
    "            time = li.xpath('./p/span[1]/text()')[-1]\n",
    "#             print(time)\n",
    "            # 回复\n",
    "            repley_num = li.xpath('./p/span[2]/text()')[-1]\n",
    "#             print(repley_num)\n",
    "            # 简介\n",
    "            synopsis = li.xpath('./div/p')[0].xpath('string(.)').strip()\n",
    "#             print(synopsis)\n",
    "            # link\n",
    "            link_url = li.xpath('./div/h3/a/@href')[-1]\n",
    "#             print(link_url)\n",
    "            \n",
    "            csv_file.writerow([title,plate,author,time,repley_num,synopsis,link_url])\n",
    "    except Exception as r:\n",
    "        print(\"获取内容异常：{}\".format(r.args))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 获取天涯搜索页面总页码，并返回urls集合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPageUrls(url):\n",
    "    urls = []\n",
    "    response = getHTMLResponse(url)\n",
    "    tree_node = etree.HTML(response.content)\n",
    "    is_exist_pages = tree_node.xpath('//div[@class=\"long-pages\"]/a')\n",
    "    # 判定是否存在多页\n",
    "    if is_exist_pages:\n",
    "        for a in is_exist_pages:\n",
    "            page_num = a.xpath('./text()')[-1]\n",
    "            if page_num.isdigit():\n",
    "                total_page = int(page_num)\n",
    "        print(\"总页数：\", total_page)\n",
    "\n",
    "        for page in range(1, total_page + 1):\n",
    "            new_url = re.sub(r'&pn=\\d+','&pn=%d' % page,url,re.S)\n",
    "            urls.append(new_url)\n",
    "    \n",
    "    # 无多页只返回原url\n",
    "    else:\n",
    "        urls.append(url)\n",
    "    \n",
    "    return urls"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. 启动天涯搜索框爬虫"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def run_Collecting_list(keyword,file_path):\n",
    "    pool = TreadPool(3)\n",
    "    contents = []\n",
    "    headers = ['title','plate','author','time','repley_num','synopsis','link_url']\n",
    "    time_start = time.time()\n",
    "    url = \"https://search.tianya.cn/bbs?q={}&pn={}&s=10\".format(keyword,1)\n",
    "    urls = getPageUrls(url)\n",
    "    \n",
    "    # 分布式获取source \n",
    "    for url,source in zip(urls,pool.map(getHTMLResponse,urls)):\n",
    "        print(url)\n",
    "        contents.append(source.content)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    \n",
    "    with open(file=file_path, mode=\"w\", newline=\"\", encoding=\"utf-8\") as file:\n",
    "        # 创建csv写入对象\n",
    "        csv_file = csv.writer(file)\n",
    "        # 写入头部信息\n",
    "        csv_file.writerow(headers)\n",
    "        # 写入数据\n",
    "        for content in contents:\n",
    "            getData(content,csv_file)\n",
    "            print('---')\n",
    "        \n",
    "    print(f'并行程耗时：{time.time()-time_start}')       "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. 使用pandas保留特定时间范围内帖子，返回urls，file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPosturls(file_list_path,start,end):\n",
    "    date_start = datetime.datetime.strptime(start, '%Y-%m-%d')\n",
    "    date_end = datetime.datetime.strptime(end, '%Y-%m-%d')\n",
    "    data = pd.read_csv(file_list_path)\n",
    "    # 除重复\n",
    "    date = data.drop_duplicates(subset=['title', 'link_url'], keep='first', inplace=True)\n",
    "    # 保留日期 转换为datetime类型\n",
    "    data['time'] = data['time'].apply(lambda x: x.split(' ')[0])\n",
    "    data['time'] = pd.to_datetime(data['time'])\n",
    "    # 筛选特定时间范围的帖子\n",
    "    target = data[(data['time'] > date_start) & ((data['time'] < date_end))]\n",
    "    filenames = target['title'].values.tolist()\n",
    "    urls = target['link_url'].values.tolist()\n",
    "    return urls,filenames"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. 抓取单个帖子函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取总页码生成新urls\n",
    "def getURLs(url):\n",
    "    urls = []\n",
    "    r = getHTMLResponse(url)\n",
    "    tree_node = etree.HTML(r.content)\n",
    "    is_exist_pages = tree_node.xpath(r'//*[@id=\"post_head\"]/div[3]/div[3]')\n",
    "    if is_exist_pages:\n",
    "        for item in tree_node.xpath(r'//div[@class=\"mb15 cf\"]/div[@class=\"atl-pages\"]//a'):\n",
    "            if item.text.isdigit():\n",
    "                total_page = int(item.text)\n",
    "        print(\"总页数：\",total_page)\n",
    "    \n",
    "        for page in range(1,total_page+1):\n",
    "            new_url = re.sub(r'-\\d+.shtml','-%d.shtml' % page,url,re.S)\n",
    "            urls.append(new_url)\n",
    "\n",
    "    # 无多页只返回原url\n",
    "    else:\n",
    "        urls.append(url)\n",
    "    return urls\n",
    "\n",
    "# 获取帖子信息\n",
    "def getDateset(r,csv_file):\n",
    "    html_doc = r.content\n",
    "    soup = BeautifulSoup(html_doc, \"html.parser\")\n",
    "    tree_node = etree.HTML(html_doc)\n",
    "    for item in soup.find_all(\"div\", class_=\"atl-item\"):\n",
    "        author = unquote(item.attrs[\"_host\"])\n",
    "        print(author)\n",
    "        \n",
    "        time = item.attrs.get(\"js_restime\")\n",
    "        if time is None:\n",
    "            author_time_info = tree_node.xpath('//*[@id=\"post_head\"]/div[2]/div[2]/span[2]/text()')[-1]\n",
    "            time = re.findall(r'时间：(.*)',author_time_info)[-1]\n",
    "#         print(time)\n",
    "        \n",
    "        content_info = item.find(\"div\", class_=\"bbs-content\")\n",
    "        content_text = content_info.text.replace('\\t','').replace('\\n','').replace('\\u3000','')\n",
    "#         print(content_text)\n",
    "\n",
    "    csv_file.writerow([author,time,content_text])\n",
    "\n",
    "# 抓取单个帖子\n",
    "def run_acquire(url,filename):\n",
    "    pool = TreadPool(3)\n",
    "    contents = []\n",
    "    intab = \"?*/\\|.:><\"\n",
    "    outtab = \"         \"\n",
    "    trantab = str.maketrans(intab, outtab)\n",
    "    FILE_PATH = '.' + os.sep + filename.translate(trantab) + '.csv'\n",
    "    headers = [\"Author\", \"Time\", \"Content\"]\n",
    "\n",
    "    time_start = time.time()\n",
    "    print(f\"开始时间：{datetime.datetime.now()}\")\n",
    "\n",
    "    urls = getURLs(url)\n",
    "\n",
    "    # 分布式获取source \n",
    "    for url,source in zip(urls,pool.map(getHTMLResponse,urls)):\n",
    "        print(url)\n",
    "        contents.append(source)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    with open(file=FILE_PATH, mode=\"w\", newline=\"\", encoding=\"utf-8\") as file:\n",
    "        # 创建csv写入对象\n",
    "        csv_file = csv.writer(file)\n",
    "        # 写入头部信息\n",
    "        csv_file.writerow(headers)\n",
    "        #\n",
    "        # 写入数据\n",
    "        for content in contents:\n",
    "            getDateset(content,csv_file)\n",
    "            print('---')\n",
    "\n",
    "    time_end = time.time()\n",
    "    print(f'结束时间：{datetime.datetime.now()}并行程耗时：{time_end-time_start}') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 主程序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    print('''\n",
    "    请输入对应数字选择模式：\n",
    "    模式1：获取关键字含帖列表\n",
    "    模式2：获取帖子信息\n",
    "    模式3：传入制定文件，获取所有帖子信息\n",
    "    输入\"1\" 选择模式1;输入\"2\"选择模式2;输入\"3\"选择模式3;按任意字符数字退出\n",
    "    ''')\n",
    "    try:\n",
    "        choose = int(input())\n",
    "    except:\n",
    "        choose = 0\n",
    "    if choose == 1:\n",
    "        keyword = \"高考 志愿 计算机 人工智能 大数据\"\n",
    "        file_path = \".\" + os.sep + keyword + '_list.csv'\n",
    "        run_Collecting_list(keyword,file_path)\n",
    "        print(\"是否要爬取特定时间范围内的所有帖子（y/n）\")\n",
    "        choice = input()\n",
    "        if choice == 'y':\n",
    "            # 输入时间\n",
    "            print('请输入开始时间:格式为xxxx-xx-xx')\n",
    "            start = input()\n",
    "            print('请输入结束时间:格式为xxxx-xx-xx')\n",
    "            end = input()\n",
    "            # 筛选特定时间\n",
    "            urls,filenames = getPosturls(file_path,start,end)\n",
    "            print(\"开始爬取......\")\n",
    "            for url,filename in zip(urls,filenames):\n",
    "                run_acquire(url,filename)\n",
    "                print(\">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")\n",
    "            print(\"Thanks use！\")\n",
    "        else:\n",
    "            print(\"Thank Use!\")  \n",
    "    elif choose == 2:\n",
    "        print(\"请输入帖子地址:\")\n",
    "        user_url = input()\n",
    "        print(\"请输入保存文件名：\")\n",
    "        user_filename = input()\n",
    "        run_acquire(user_url,user_filename)\n",
    "        print(\"Thank Use!\")\n",
    "    elif choose == 3:\n",
    "        print(\"请输入文件路径：\")\n",
    "        file_list_path = input()\n",
    "        data = pd.read_csv(file_list_path)\n",
    "        # 除重复\n",
    "        date = data.drop_duplicates(subset=['title', 'link_url'], keep='first', inplace=True)\n",
    "        filenames = date['title'].values.tolist()\n",
    "        urls = date['link_url'].values.tolist()\n",
    "        for url,filename in zip(urls,filenames):\n",
    "             run_acquire(url,filename)\n",
    "        print(\"Thank Use!\")\n",
    "    else:\n",
    "        print(\"Thank Use!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    请输入对应数字选择模式：\n",
      "    模式1：获取关键字含帖列表\n",
      "    模式2：获取帖子信息\n",
      "    模式3：传入制定文件，获取所有帖子信息\n",
      "    输入\"1\" 选择模式1;输入\"2\"选择模式2;输入\"3\"选择模式3;按任意字符数字退出\n",
      "    \n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://search.tianya.cn/bbs?q=高考 志愿 计算机 人工智能 大数据&pn=1&s=10\n",
      "高考填志愿:千万不能错过这些专业\n",
      "北大清华网站3篇论文分获点击数双状元:江西中行北大学士高级统计师刘平(转载)\n",
      "欧雷|人工智能,助飞未来教育(转载)\n",
      "---\n",
      "并行程耗时：1.022266149520874\n",
      "是否要爬取特定时间范围内的所有帖子（y/n）\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " y\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "请输入开始时间:格式为xxxx-xx-xx\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " 2000-01-01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "请输入结束时间:格式为xxxx-xx-xx\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " 2020-12-31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "开始爬取......\n",
      "开始时间：2020-08-07 00:23:29.723412\n",
      "http://bbs.tianya.cn/post-university-715049-1.shtml\n",
      "AmIgreat\n",
      "---\n",
      "结束时间：2020-08-07 00:23:30.674867并行程耗时：0.9514555931091309\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "开始时间：2020-08-07 00:23:30.676862\n",
      "http://bbs.tianya.cn/post-no01-521692-1.shtml\n",
      "liupingzgr\n",
      "---\n",
      "结束时间：2020-08-07 00:23:32.055805并行程耗时：1.3789432048797607\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "开始时间：2020-08-07 00:23:32.057800\n",
      "http://bbs.tianya.cn/post-itinfo-528697-1.shtml\n",
      "yufengzuoban\n",
      "---\n",
      "结束时间：2020-08-07 00:23:33.105996并行程耗时：1.0481960773468018\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Thanks use！\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
